package com.syl.qianyin.spider.service;

import com.syl.qianyin.spider.exend.CustomHttpClientDownloader;
import com.syl.qianyin.spider.exend.interfaces.CustomPageProcessor;
import org.jsoup.nodes.Document;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.selector.Html;

/**
 * @author syl
 * @create 2019-05-12 14:35
 **/
@Service
public class XiaTest implements CustomPageProcessor {

    @Override
    public void myProcess(Page page, String url, Html html, Document document) {
//        System.out.println(html.get());
    }

    @Override
    public void start() {
        CustomHttpClientDownloader httpClientDownloader = new CustomHttpClientDownloader(
//            SimpleProxyProvider.from(
//                new Proxy("112.85.128.74",9999),
//                new Proxy("112.87.68.39",9999),
//                new Proxy("112.85.128.173",9999),
//                new Proxy("112.87.69.148",9999)
//            )
        );
        Spider.create(this)
            .addUrl("http://localhost/guidepost/")
            .setDownloader(httpClientDownloader)
            .addPipeline(new ConsolePipeline())
            .run();
    }

}
